package com.db;

import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Map;

import org.mapdb.DB;
import org.mapdb.DBMaker;

import com.access.Crawler;
import com.data.JDoc;

public class Indexer {

	public static void main(String[] args) throws FileNotFoundException {

		Crawler crawler = new Crawler();
		crawler.crawl(
				"http://www.cse.ust.hk/~ericzhao/COMP4321/TestPages/testpage.htm",
				300);

		DB db = DBMaker.newFileDB(new File("data/web.db")).closeOnJvmShutdown()
				.make();
		Map<String, Integer> urlToDocId = db.getHashMap("urlToDocId");
		Map<Integer, JDoc> idToDoc = db.getHashMap("idToDoc");
		Map<String, Map<Integer, Float>> invertedIndex = db
				.getHashMap("invertedIndex");

		System.out.println("\nfinish initializing db");

		urlToDocId.putAll(crawler.getUrlToDocId());
		idToDoc.putAll(crawler.getIdToDoc());
		invertedIndex.putAll(crawler.getInvertedIndex());

		System.out.println("\nfinish building table");

		db.commit();
		db.close();

		System.out.println("\nfinish storing db");

		System.out.println("\n------------------------------");
		System.out.println("start testing db");

		DB testDB = DBMaker.newFileDB(new File("data/web.db"))
				.closeOnJvmShutdown().make();

		Map<String, Integer> testUrlToDocId = testDB.getHashMap("urlToDocId");
		Map<String, Date> testUrlToUpdate = testDB.getHashMap("urlToUpdate");
		Map<Integer, JDoc> testIdToDoc = testDB.getHashMap("idToDoc");
		Map<String, Map<Integer, Float>> testInvertedIndex = testDB
				.getHashMap("invertedIndex");

		System.out.println("\nfinish fetching data");

		System.out.println("\ntest table : url -> doc id (size : "
				+ testUrlToDocId.size() + ")");
		int i = 0;
		for (String url : testUrlToDocId.keySet()) {
			System.out.println(url + " -> " + testUrlToDocId.get(url));
			i++;
			if (i > 19)
				break;
		}

		System.out.println("\ntest table : url -> last modified date (size : "
				+ testUrlToUpdate.size() + ")");
		i = 0;
		for (String url : testUrlToUpdate.keySet()) {
			System.out.println(url + " -> " + testUrlToUpdate.get(url));
			i++;
			if (i > 19)
				break;
		}

		System.out.println("\ntest table : doc id -> doc (size : "
				+ testIdToDoc.size() + ")");
		i = 0;
		for (Integer docId : testIdToDoc.keySet()) {
			int k = 0;
			JDoc doc = testIdToDoc.get(docId);

			System.out.println(docId + " -> " + doc.getTitle());

			System.out.println(doc.getUrl());

			System.out.println(doc.getLastModified() + " (size : "
					+ doc.getSize() + ")");

			Map<String, Integer> wordList = doc.getWords();
			k = 0;
			for (String term : wordList.keySet()) {
				System.out.print("(" + term + ", " + wordList.get(term) + ") ");
				if (k >= 19) {
					break;
				}
				k++;
			}

			System.out.print("\n");

			ArrayList<String> parentLinks = doc.getParentUrls();
			k = 0;
			System.out.println("parent urls :");
			while (k < 19 && k < parentLinks.size()) {
				System.out.println(parentLinks.get(k));
				k++;
			}

			ArrayList<String> childLinks = doc.getChildLinks();
			k = 0;
			System.out.println("child urls :");
			while (k < 19 && k < childLinks.size()) {
				System.out.println(childLinks.get(k));
				k++;
			}
			System.out.println();

			i++;
			if (i > 19)
				break;
		}

		System.out.println("\ntest table : term -> (doc, tw) (size : "
				+ testInvertedIndex.size() + ")");
		i = 0;
		for (String term : testInvertedIndex.keySet()) {
			Map<Integer, Float> docTwList = testInvertedIndex.get(term);
			System.out.print(term + " -> ");

			for (Integer docId : docTwList.keySet()) {
				System.out.print("(" + docId + ", " + docTwList.get(docId)
						+ ")  ");
			}
			System.out.print("\n");
			i++;
			if (i > 19)
				break;
		}

		System.out.println("\nfinish testing");

		System.out.println("\ntesting doc 0");

		int k = 0;
		JDoc doc = testIdToDoc.get(0);

		System.out.println(0 + " -> " + doc.getTitle());

		System.out.println(doc.getUrl());

		System.out.println(doc.getLastModified() + " (size : " + doc.getSize()
				+ ")");

		Map<String, Integer> wordList = doc.getWords();
		k = 0;
		for (String term : wordList.keySet()) {
			System.out.print("(" + term + ", " + wordList.get(term) + ") ");
			if (k >= 19) {
				break;
			}
			k++;
		}

		System.out.print("\n");

		ArrayList<String> parentLinks = doc.getParentUrls();
		k = 0;
		System.out.println("parent urls :");
		while (k < 19 && k < parentLinks.size()) {
			System.out.println(parentLinks.get(k));
			k++;
		}

		ArrayList<String> childLinks = doc.getChildLinks();
		k = 0;
		System.out.println("child urls :");
		while (k < 19 && k < childLinks.size()) {
			System.out.println(childLinks.get(k));
			k++;
		}

		ArrayList<String> stemmedTitle = doc.getStemmedTitle();
		k = 0;
		System.out.println("stemmedTitle :");
		while (k < 19 && k < stemmedTitle.size()) {
			System.out.println(stemmedTitle.get(k));
			k++;
		}

		ArrayList<String> stemmedBody = doc.getStemmedBody();
		k = 0;
		System.out.println("stemmedBody :");
		while (k < 19 && k < stemmedBody.size()) {
			System.out.println(stemmedBody.get(k));
			k++;
		}
	}
}
